"""
This crawler crawling the 2dfan,a website introducing galageme information
"""
# coding:utf-8
from urllib.request import urlopen
from urllib.request import Request
from urllib.error import URLError, HTTPError
from bs4 import BeautifulSoup
import re


indexUrl = "http://www.bilibili.com/video/bangumi_chinese_1.html"
index = 5169271
#errorTimes = 0
#tolerateErrorTimes = 50
url = "http://www.bilibili.com/"
user_agent = "Mozilla/5.0"
headers = {'User-Agent': user_agent}
# test the webpage openning
try:
    entireUrl = "http://www.bilibili.com/video/bangumi_chinese_1.html"
    webPage = Request(entireUrl, None, headers)
    html = urlopen(webPage)
except URLError as e1:
    print("url open ERROR!,reason is %s" %(e1.reason))
except HTTPError as e2:
    print("url open ERROR!,error code is %d,reason is %s,header is%s" %(e2.code, e2.reason, e2.headers))

bsObj = BeautifulSoup(html, "html.parser")

# crawl content
try:
    content = bsObj.find_all("div")
    print("%s" %(content.prettify()))
    errorTimes = 0
except AttributeError as e:
    print("ERROR!,reason is %s" % (e.reason))

print("COMPLETE!TOTAL: %d OBJECTS" %(errorTimes))